{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35000"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv('image_train_dataset.tsv', delimiter='\\t')\n",
    "total_samples = train_df.shape[0]\n",
    "total_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_captions(caption_list):\n",
    "    pc = []\n",
    "    for caption in caption_list:\n",
    "        caption = caption.strip().lower()\n",
    "        caption = caption.replace('.', '').replace(',', '').replace(\"'\", \"\").replace('\"', '')\n",
    "        caption = caption.replace('&','and').replace('(','').replace(')', '').replace('-', ' ')\n",
    "        caption = ' '.join(caption.split()) \n",
    "        caption = '<START> '+caption+' <END>'\n",
    "        pc.append(caption)\n",
    "    return pc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_captions = train_df.caption.tolist()\n",
    "processed_train_captions = preprocess_captions(train_captions)\n",
    "\n",
    "tc_tokens = [caption.split() for caption in processed_train_captions]\n",
    "tc_tokens_length = [len(tokenized_caption) for tokenized_caption in tc_tokens]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "tc_words = [word.strip() for word_list in tc_tokens for word in word_list]\n",
    "unique_words = list(set(tc_words))\n",
    "token_counter = Counter(unique_words)\n",
    "\n",
    "word_to_index = {item[0]: index+1 for index, item in enumerate(dict(token_counter).items())}\n",
    "word_to_index['<PAD>'] = 0\n",
    "index_to_word = {index: word for word, index in word_to_index.items()}\n",
    "vocab_size = len(word_to_index)\n",
    "max_caption_size = np.max(tc_tokens_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['vocabulary_metadata.pkl']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.externals import joblib\n",
    "\n",
    "vocab_metadata = dict()\n",
    "vocab_metadata['word2index'] = word_to_index\n",
    "vocab_metadata['index2word'] = index_to_word\n",
    "vocab_metadata['max_caption_size'] = max_caption_size\n",
    "vocab_metadata['vocab_size'] = vocab_size\n",
    "\n",
    "joblib.dump(vocab_metadata, 'vocabulary_metadata.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'index2word': [(0, '<PAD>'),\n",
       "  (1, 'nearby'),\n",
       "  (2, 'flooded'),\n",
       "  (3, 'fundraising'),\n",
       "  (4, 'snowboarder')],\n",
       " 'max_caption_size': 39,\n",
       " 'vocab_size': 7927,\n",
       " 'word2index': [('reflections', 4122),\n",
       "  ('flakes', 1829),\n",
       "  ('flexing', 7684),\n",
       "  ('scaling', 1057),\n",
       "  ('pretend', 6788)]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "{k: v if type(v) is not dict \n",
    "         else list(v.items())[:5] \n",
    "             for k, v in vocab_metadata.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<START> a black dog is running after a white dog in the snow <END>'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_train_captions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.externals import joblib\n",
    "\n",
    "tl_img_feature_map = joblib.load('transfer_learn_img_features.pkl')\n",
    "vocab_metadata = joblib.load('vocabulary_metadata.pkl')\n",
    "\n",
    "\n",
    "train_img_names = train_df.image.tolist()\n",
    "train_img_features = [tl_img_feature_map[img_name] for img_name in train_img_names]\n",
    "train_img_features = np.array(train_img_features)\n",
    "\n",
    "word_to_index = vocab_metadata['word2index']\n",
    "index_to_word = vocab_metadata['index2word'] \n",
    "max_caption_size = vocab_metadata['max_caption_size']\n",
    "vocab_size = vocab_metadata['vocab_size']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(35000, 4096)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_img_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "\n",
    "def dataset_generator(processed_captions, transfer_learnt_features, vocab_size, max_caption_size, batch_size=32):\n",
    "    partial_caption_set = []\n",
    "    next_word_seq_set = []\n",
    "    img_feature_set = []\n",
    "    batch_count = 0\n",
    "    batch_num = 0\n",
    "    \n",
    "    while True:\n",
    "        for index, caption in enumerate(processed_captions):\n",
    "            img_features = transfer_learnt_features[index]\n",
    "            for cap_idx in range(len(caption.split()) - 1):\n",
    "                partial_caption = [word_to_index[word] for word in caption.split()[:cap_idx+1]]\n",
    "                partial_caption_set.append(partial_caption)\n",
    "\n",
    "                next_word_seq = np.zeros(vocab_size)\n",
    "                next_word_seq[word_to_index[caption.split()[cap_idx+1]]] = 1\n",
    "                next_word_seq_set.append(next_word_seq)\n",
    "                img_feature_set.append(img_features)\n",
    "                batch_count+=1\n",
    "\n",
    "                if batch_count >= batch_size:\n",
    "                    batch_num += 1\n",
    "                    img_feature_set = np.array(img_feature_set)\n",
    "                    partial_caption_set = sequence.pad_sequences(sequences=partial_caption_set, maxlen=max_caption_size, \n",
    "                                                                 padding='post')\n",
    "                    next_word_seq_set = np.array(next_word_seq_set)\n",
    "                    \n",
    "                    yield [[img_feature_set, partial_caption_set], next_word_seq_set]\n",
    "                    batch_count = 0\n",
    "                    partial_caption_set = []\n",
    "                    next_word_seq_set = []\n",
    "                    img_feature_set = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size: 7927\n",
      "Max caption size: 39\n",
      "Test Batch size: 10\n"
     ]
    }
   ],
   "source": [
    "MAX_CAPTION_SIZE = max_caption_size\n",
    "VOCABULARY_SIZE = vocab_size\n",
    "BATCH_SIZE = 10\n",
    "\n",
    "print('Vocab size:', VOCABULARY_SIZE)\n",
    "print('Max caption size:', MAX_CAPTION_SIZE)\n",
    "print('Test Batch size:', BATCH_SIZE)\n",
    "\n",
    "d = dataset_generator(processed_captions=processed_train_captions, \n",
    "                  transfer_learnt_features=train_img_features, \n",
    "                  vocab_size=VOCABULARY_SIZE, max_caption_size=MAX_CAPTION_SIZE,\n",
    "                  batch_size=BATCH_SIZE)\n",
    "d = list(d)\n",
    "img_features, partial_captions = d[0][0]\n",
    "next_word = d[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((10, 4096), (10, 39), (10, 7927))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_features.shape, partial_captions.shape, next_word.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ],\n",
       "       ...,\n",
       "       [0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.704, ..., 0.   , 0.   , 0.   ]], dtype=float32)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round(img_features, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6917    0    0    0    0    0    0    0    0    0    0]\n",
      " [6917 2578    0    0    0    0    0    0    0    0    0]\n",
      " [6917 2578 7371    0    0    0    0    0    0    0    0]\n",
      " [6917 2578 7371 3519    0    0    0    0    0    0    0]\n",
      " [6917 2578 7371 3519 3113    0    0    0    0    0    0]\n",
      " [6917 2578 7371 3519 3113 6720    0    0    0    0    0]\n",
      " [6917 2578 7371 3519 3113 6720    7    0    0    0    0]\n",
      " [6917 2578 7371 3519 3113 6720    7 2578    0    0    0]\n",
      " [6917 2578 7371 3519 3113 6720    7 2578 1076    0    0]\n",
      " [6917 2578 7371 3519 3113 6720    7 2578 1076 3519    0]]\n"
     ]
    }
   ],
   "source": [
    "print(np.array([partial_caption[:11] for partial_caption in partial_captions]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['<START>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' '<PAD>' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' '<PAD>' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' 'running' '<PAD>' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' 'running' 'after' '<PAD>' '<PAD>'\n",
      "  '<PAD>' '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' 'running' 'after' 'a' '<PAD>' '<PAD>'\n",
      "  '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' 'running' 'after' 'a' 'white' '<PAD>'\n",
      "  '<PAD>']\n",
      " ['<START>' 'a' 'black' 'dog' 'is' 'running' 'after' 'a' 'white' 'dog'\n",
      "  '<PAD>']]\n"
     ]
    }
   ],
   "source": [
    "print(np.array([[index_to_word[word] for word in cap][:11] for cap in partial_captions]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Next word positions: [2578 7371 3519 3113 6720    7 2578 1076 3519 5070]\n",
      "Next words: ['a', 'black', 'dog', 'is', 'running', 'after', 'a', 'white', 'dog', 'in']\n"
     ]
    }
   ],
   "source": [
    "print('Next word positions:', np.nonzero(next_word)[1])\n",
    "print('Next words:', [index_to_word[word] for word in np.nonzero(next_word)[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers import LSTM, Embedding, TimeDistributed, Dense, RepeatVector, Activation, Flatten, concatenate\n",
    "\n",
    "DENSE_DIM = 256\n",
    "EMBEDDING_DIM = 256\n",
    "MAX_CAPTION_SIZE = max_caption_size\n",
    "VOCABULARY_SIZE = vocab_size\n",
    "\n",
    "image_model = Sequential()\n",
    "image_model.add(Dense(DENSE_DIM, input_dim=4096, activation='relu'))\n",
    "image_model.add(RepeatVector(MAX_CAPTION_SIZE))\n",
    "\n",
    "language_model = Sequential()\n",
    "language_model.add(Embedding(VOCABULARY_SIZE, EMBEDDING_DIM, input_length=MAX_CAPTION_SIZE))\n",
    "language_model.add(LSTM(256, return_sequences=True))\n",
    "language_model.add(TimeDistributed(Dense(DENSE_DIM)))\n",
    "\n",
    "merged_output = concatenate([image_model.output, language_model.output])\n",
    "merged_output = LSTM(1024, return_sequences=False)(merged_output)\n",
    "merged_output = (Dense(VOCABULARY_SIZE))(merged_output)\n",
    "merged_output = Activation('softmax')(merged_output)\n",
    "\n",
    "model = Model([image_model.input, language_model.input], merged_output)\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "embedding_1_input (InputLayer)   (None, 39)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "dense_1_input (InputLayer)       (None, 4096)          0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 39, 256)       2029312     embedding_1_input[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 256)           1048832     dense_1_input[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                    (None, 39, 256)       525312      embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "repeat_vector_1 (RepeatVector)   (None, 39, 256)       0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 39, 256)       65792       lstm_1[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 39, 512)       0           repeat_vector_1[0][0]            \n",
      "                                                                   time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                    (None, 1024)          6295552     concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 7927)          8125175     lstm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "activation_1 (Activation)        (None, 7927)          0           dense_3[0][0]                    \n",
      "====================================================================================================\n",
      "Total params: 18,089,975\n",
      "Trainable params: 18,089,975\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import ReduceLROnPlateau\n",
    "\n",
    "reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.15,\n",
    "                              patience=2, min_lr=0.000005)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"636pt\" viewBox=\"0.00 0.00 590.00 636.00\" width=\"590pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 632)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-632 586,-632 586,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 1914517544408 -->\n",
       "<g class=\"node\" id=\"node1\"><title>1914517544408</title>\n",
       "<polygon fill=\"none\" points=\"50,-581.5 50,-627.5 260,-627.5 260,-581.5 50,-581.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"88.5\" y=\"-600.8\">InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"127,-581.5 127,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"155\" y=\"-612.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"127,-604.5 183,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"155\" y=\"-589.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"183,-581.5 183,-627.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"221.5\" y=\"-612.3\">(None, 39)</text>\n",
       "<polyline fill=\"none\" points=\"183,-604.5 260,-604.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"221.5\" y=\"-589.3\">(None, 39)</text>\n",
       "</g>\n",
       "<!-- 1914517543736 -->\n",
       "<g class=\"node\" id=\"node3\"><title>1914517543736</title>\n",
       "<polygon fill=\"none\" points=\"35,-498.5 35,-544.5 275,-544.5 275,-498.5 35,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"75\" y=\"-517.8\">Embedding</text>\n",
       "<polyline fill=\"none\" points=\"115,-498.5 115,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"143\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"115,-521.5 171,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"143\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"171,-498.5 171,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"223\" y=\"-529.3\">(None, 39)</text>\n",
       "<polyline fill=\"none\" points=\"171,-521.5 275,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"223\" y=\"-506.3\">(None, 39, 256)</text>\n",
       "</g>\n",
       "<!-- 1914517544408&#45;&gt;1914517543736 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>1914517544408-&gt;1914517543736</title>\n",
       "<path d=\"M155,-581.366C155,-573.152 155,-563.658 155,-554.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"158.5,-554.607 155,-544.607 151.5,-554.607 158.5,-554.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914517543400 -->\n",
       "<g class=\"node\" id=\"node2\"><title>1914517543400</title>\n",
       "<polygon fill=\"none\" points=\"343.5,-498.5 343.5,-544.5 566.5,-544.5 566.5,-498.5 343.5,-498.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"382\" y=\"-517.8\">InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"420.5,-498.5 420.5,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"448.5\" y=\"-529.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"420.5,-521.5 476.5,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"448.5\" y=\"-506.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"476.5,-498.5 476.5,-544.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"521.5\" y=\"-529.3\">(None, 4096)</text>\n",
       "<polyline fill=\"none\" points=\"476.5,-521.5 566.5,-521.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"521.5\" y=\"-506.3\">(None, 4096)</text>\n",
       "</g>\n",
       "<!-- 1914517542728 -->\n",
       "<g class=\"node\" id=\"node4\"><title>1914517542728</title>\n",
       "<polygon fill=\"none\" points=\"356.5,-415.5 356.5,-461.5 553.5,-461.5 553.5,-415.5 356.5,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"382\" y=\"-434.8\">Dense</text>\n",
       "<polyline fill=\"none\" points=\"407.5,-415.5 407.5,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"435.5\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"407.5,-438.5 463.5,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"435.5\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"463.5,-415.5 463.5,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"508.5\" y=\"-446.3\">(None, 4096)</text>\n",
       "<polyline fill=\"none\" points=\"463.5,-438.5 553.5,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"508.5\" y=\"-423.3\">(None, 256)</text>\n",
       "</g>\n",
       "<!-- 1914517543400&#45;&gt;1914517542728 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>1914517543400-&gt;1914517542728</title>\n",
       "<path d=\"M455,-498.366C455,-490.152 455,-480.658 455,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"458.5,-471.607 455,-461.607 451.5,-471.607 458.5,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914517543960 -->\n",
       "<g class=\"node\" id=\"node5\"><title>1914517543960</title>\n",
       "<polygon fill=\"none\" points=\"48,-415.5 48,-461.5 262,-461.5 262,-415.5 48,-415.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"75\" y=\"-434.8\">LSTM</text>\n",
       "<polyline fill=\"none\" points=\"102,-415.5 102,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"130\" y=\"-446.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"102,-438.5 158,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"130\" y=\"-423.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"158,-415.5 158,-461.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210\" y=\"-446.3\">(None, 39, 256)</text>\n",
       "<polyline fill=\"none\" points=\"158,-438.5 262,-438.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210\" y=\"-423.3\">(None, 39, 256)</text>\n",
       "</g>\n",
       "<!-- 1914517543736&#45;&gt;1914517543960 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>1914517543736-&gt;1914517543960</title>\n",
       "<path d=\"M155,-498.366C155,-490.152 155,-480.658 155,-471.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"158.5,-471.607 155,-461.607 151.5,-471.607 158.5,-471.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914517543008 -->\n",
       "<g class=\"node\" id=\"node6\"><title>1914517543008</title>\n",
       "<polygon fill=\"none\" points=\"328,-332.5 328,-378.5 582,-378.5 582,-332.5 328,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"375\" y=\"-351.8\">RepeatVector</text>\n",
       "<polyline fill=\"none\" points=\"422,-332.5 422,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"450\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"422,-355.5 478,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"450\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"478,-332.5 478,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"530\" y=\"-363.3\">(None, 256)</text>\n",
       "<polyline fill=\"none\" points=\"478,-355.5 582,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"530\" y=\"-340.3\">(None, 39, 256)</text>\n",
       "</g>\n",
       "<!-- 1914517542728&#45;&gt;1914517543008 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>1914517542728-&gt;1914517543008</title>\n",
       "<path d=\"M455,-415.366C455,-407.152 455,-397.658 455,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"458.5,-388.607 455,-378.607 451.5,-388.607 458.5,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914734920816 -->\n",
       "<g class=\"node\" id=\"node7\"><title>1914734920816</title>\n",
       "<polygon fill=\"none\" points=\"0,-332.5 0,-378.5 310,-378.5 310,-332.5 0,-332.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"75\" y=\"-351.8\">TimeDistributed(Dense)</text>\n",
       "<polyline fill=\"none\" points=\"150,-332.5 150,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"178\" y=\"-363.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"150,-355.5 206,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"178\" y=\"-340.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"206,-332.5 206,-378.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-363.3\">(None, 39, 256)</text>\n",
       "<polyline fill=\"none\" points=\"206,-355.5 310,-355.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"258\" y=\"-340.3\">(None, 39, 256)</text>\n",
       "</g>\n",
       "<!-- 1914517543960&#45;&gt;1914734920816 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>1914517543960-&gt;1914734920816</title>\n",
       "<path d=\"M155,-415.366C155,-407.152 155,-397.658 155,-388.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"158.5,-388.607 155,-378.607 151.5,-388.607 158.5,-388.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914517759368 -->\n",
       "<g class=\"node\" id=\"node8\"><title>1914517759368</title>\n",
       "<polygon fill=\"none\" points=\"129.5,-249.5 129.5,-295.5 480.5,-295.5 480.5,-249.5 129.5,-249.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"172.5\" y=\"-268.8\">Concatenate</text>\n",
       "<polyline fill=\"none\" points=\"215.5,-249.5 215.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"243.5\" y=\"-280.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"215.5,-272.5 271.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"243.5\" y=\"-257.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"271.5,-249.5 271.5,-295.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"376\" y=\"-280.3\">[(None, 39, 256), (None, 39, 256)]</text>\n",
       "<polyline fill=\"none\" points=\"271.5,-272.5 480.5,-272.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"376\" y=\"-257.3\">(None, 39, 512)</text>\n",
       "</g>\n",
       "<!-- 1914517543008&#45;&gt;1914517759368 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>1914517543008-&gt;1914517759368</title>\n",
       "<path d=\"M414.012,-332.366C395.74,-322.5 374.052,-310.788 354.851,-300.419\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"356.401,-297.279 345.939,-295.607 353.075,-303.438 356.401,-297.279\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914734920816&#45;&gt;1914517759368 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>1914734920816-&gt;1914517759368</title>\n",
       "<path d=\"M195.988,-332.366C214.26,-322.5 235.948,-310.788 255.149,-300.419\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"256.925,-303.438 264.061,-295.607 253.599,-297.279 256.925,-303.438\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914734842040 -->\n",
       "<g class=\"node\" id=\"node9\"><title>1914734842040</title>\n",
       "<polygon fill=\"none\" points=\"198,-166.5 198,-212.5 412,-212.5 412,-166.5 198,-166.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"225\" y=\"-185.8\">LSTM</text>\n",
       "<polyline fill=\"none\" points=\"252,-166.5 252,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"280\" y=\"-197.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"252,-189.5 308,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"280\" y=\"-174.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"308,-166.5 308,-212.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"360\" y=\"-197.3\">(None, 39, 512)</text>\n",
       "<polyline fill=\"none\" points=\"308,-189.5 412,-189.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"360\" y=\"-174.3\">(None, 1024)</text>\n",
       "</g>\n",
       "<!-- 1914517759368&#45;&gt;1914734842040 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>1914517759368-&gt;1914734842040</title>\n",
       "<path d=\"M305,-249.366C305,-241.152 305,-231.658 305,-222.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"308.5,-222.607 305,-212.607 301.5,-222.607 308.5,-222.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914734793392 -->\n",
       "<g class=\"node\" id=\"node10\"><title>1914734793392</title>\n",
       "<polygon fill=\"none\" points=\"206.5,-83.5 206.5,-129.5 403.5,-129.5 403.5,-83.5 206.5,-83.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"232\" y=\"-102.8\">Dense</text>\n",
       "<polyline fill=\"none\" points=\"257.5,-83.5 257.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"285.5\" y=\"-114.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"257.5,-106.5 313.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"285.5\" y=\"-91.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"313.5,-83.5 313.5,-129.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"358.5\" y=\"-114.3\">(None, 1024)</text>\n",
       "<polyline fill=\"none\" points=\"313.5,-106.5 403.5,-106.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"358.5\" y=\"-91.3\">(None, 7927)</text>\n",
       "</g>\n",
       "<!-- 1914734842040&#45;&gt;1914734793392 -->\n",
       "<g class=\"edge\" id=\"edge9\"><title>1914734842040-&gt;1914734793392</title>\n",
       "<path d=\"M305,-166.366C305,-158.152 305,-148.658 305,-139.725\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"308.5,-139.607 305,-129.607 301.5,-139.607 308.5,-139.607\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 1914735097280 -->\n",
       "<g class=\"node\" id=\"node11\"><title>1914735097280</title>\n",
       "<polygon fill=\"none\" points=\"195,-0.5 195,-46.5 415,-46.5 415,-0.5 195,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"232\" y=\"-19.8\">Activation</text>\n",
       "<polyline fill=\"none\" points=\"269,-0.5 269,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"297\" y=\"-31.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"269,-23.5 325,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"297\" y=\"-8.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"325,-0.5 325,-46.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"370\" y=\"-31.3\">(None, 7927)</text>\n",
       "<polyline fill=\"none\" points=\"325,-23.5 415,-23.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"370\" y=\"-8.3\">(None, 7927)</text>\n",
       "</g>\n",
       "<!-- 1914734793392&#45;&gt;1914735097280 -->\n",
       "<g class=\"edge\" id=\"edge10\"><title>1914734793392-&gt;1914735097280</title>\n",
       "<path d=\"M305,-83.3664C305,-75.1516 305,-65.6579 305,-56.7252\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"308.5,-56.6068 305,-46.6068 301.5,-56.6069 308.5,-56.6068\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "SVG(model_to_dot(model, show_shapes=True, show_layer_names=False, \n",
    "    rankdir='TB').create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 4.1236 - acc: 0.2823\n",
      "Epoch 2/30\n",
      "1617/1617 [==============================] - 725s 448ms/step - loss: 3.9182 - acc: 0.3150\n",
      "Epoch 3/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.8286 - acc: 0.3281\n",
      "Epoch 4/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.8102 - acc: 0.3369\n",
      "Epoch 5/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.7773 - acc: 0.3446\n",
      "Epoch 6/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.7513 - acc: 0.3500\n",
      "Epoch 7/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.7250 - acc: 0.3533\n",
      "Epoch 8/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.7613 - acc: 0.3568\n",
      "Epoch 9/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.7421 - acc: 0.3593\n",
      "Epoch 10/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6979 - acc: 0.3635\n",
      "Epoch 11/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6917 - acc: 0.3655\n",
      "Epoch 12/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6911 - acc: 0.3668\n",
      "Epoch 13/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6864 - acc: 0.3695\n",
      "Epoch 14/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6922 - acc: 0.3711\n",
      "Epoch 15/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6616 - acc: 0.3734\n",
      "Epoch 16/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6630 - acc: 0.3745\n",
      "Epoch 17/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6568 - acc: 0.3773\n",
      "Epoch 18/30\n",
      "1617/1617 [==============================] - 723s 447ms/step - loss: 3.6480 - acc: 0.3785\n",
      "Epoch 19/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6359 - acc: 0.3817\n",
      "Epoch 20/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6497 - acc: 0.3817\n",
      "Epoch 21/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6470 - acc: 0.3835\n",
      "Epoch 22/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6218 - acc: 0.3846\n",
      "Epoch 23/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6293 - acc: 0.3865\n",
      "Epoch 24/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6319 - acc: 0.3874\n",
      "Epoch 25/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6165 - acc: 0.3886\n",
      "Epoch 26/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6101 - acc: 0.3893\n",
      "Epoch 27/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6177 - acc: 0.3897\n",
      "Epoch 28/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.6347 - acc: 0.3906\n",
      "Epoch 29/30\n",
      "1617/1617 [==============================] - 724s 447ms/step - loss: 3.6443 - acc: 0.3885\n",
      "Epoch 30/30\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.4656 - acc: 0.4078\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 256\n",
    "EPOCHS = 30\n",
    "cap_lens = [(cl-1) for cl in tc_tokens_length]\n",
    "total_size = sum(cap_lens)\n",
    "\n",
    "history = model.fit_generator(dataset_generator(processed_captions=processed_train_captions, \n",
    "                                                transfer_learnt_features=train_img_features, \n",
    "                                                vocab_size=VOCABULARY_SIZE, max_caption_size=MAX_CAPTION_SIZE,\n",
    "                                                batch_size=BATCH_SIZE), \n",
    "                              steps_per_epoch=int(total_size/BATCH_SIZE), \n",
    "                              callbacks=[reduce_lr],\n",
    "                              epochs=EPOCHS, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('ic_model_rmsprop_b256ep30.h5') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31/50\n",
      "1617/1617 [==============================] - 724s 447ms/step - loss: 3.3988 - acc: 0.4144\n",
      "Epoch 32/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.3633 - acc: 0.4184\n",
      "Epoch 33/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.3374 - acc: 0.4221\n",
      "Epoch 34/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.3186 - acc: 0.4249\n",
      "Epoch 35/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2979 - acc: 0.4275\n",
      "Epoch 36/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2795 - acc: 0.4298\n",
      "Epoch 37/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2644 - acc: 0.4321\n",
      "Epoch 38/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2503 - acc: 0.4342\n",
      "Epoch 39/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2368 - acc: 0.4358\n",
      "Epoch 40/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2248 - acc: 0.4376\n",
      "Epoch 41/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.2126 - acc: 0.4391\n",
      "Epoch 42/50\n",
      "1617/1617 [==============================] - 723s 447ms/step - loss: 3.2014 - acc: 0.4407\n",
      "Epoch 43/50\n",
      "1617/1617 [==============================] - 723s 447ms/step - loss: 3.1886 - acc: 0.4421\n",
      "Epoch 44/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.1786 - acc: 0.4444\n",
      "Epoch 45/50\n",
      "1617/1617 [==============================] - 724s 447ms/step - loss: 3.1703 - acc: 0.4457\n",
      "Epoch 46/50\n",
      "1617/1617 [==============================] - 724s 447ms/step - loss: 3.1616 - acc: 0.4468\n",
      "Epoch 47/50\n",
      "1617/1617 [==============================] - 724s 447ms/step - loss: 3.1511 - acc: 0.4480\n",
      "Epoch 48/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.1420 - acc: 0.4494\n",
      "Epoch 49/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.1330 - acc: 0.4509\n",
      "Epoch 50/50\n",
      "1617/1617 [==============================] - 724s 448ms/step - loss: 3.1260 - acc: 0.4523\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 50\n",
    "\n",
    "history_rest = model.fit_generator(dataset_generator(processed_captions=processed_train_captions, \n",
    "                                                transfer_learnt_features=train_img_features, \n",
    "                                                vocab_size=VOCABULARY_SIZE, max_caption_size=MAX_CAPTION_SIZE,\n",
    "                                                batch_size=BATCH_SIZE), \n",
    "                              steps_per_epoch=int(total_size/BATCH_SIZE), \n",
    "                              callbacks=[reduce_lr],\n",
    "                              epochs=EPOCHS, verbose=1, initial_epoch=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('ic_model_rmsprop_b256ep50.h5') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "epochs = list(range(1,51))\n",
    "losses = history.history['loss'] + history_rest.history['loss'] \n",
    "accs = history.history['acc'] + history_rest.history['acc'] \n",
    "lrs = history.history['lr'] + history_rest.history['lr'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.text.Text at 0x7f599d8707b8>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0MAAAEjCAYAAADwq9qcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XecVNX9//HXZ2YbZVl670WRogQR1NhLRKOQRJNYYu/R\naIqJJcb4M/H7jX7z1RRLvvaKPRqiGGMjdjqI9N5h6Uvb/vn9ce/iuG4Ddpi7s+/n47EP5t4598zn\ngs7OZ845n2PujoiIiIiISGMTS3UAIiIiIiIiqaBkSEREREREGiUlQyIiIiIi0igpGRIRERERkUZJ\nyZCIiIiIiDRKSoZERERERKRRUjIkItKAmVlbM3MzO24PrrndzL5IYlg1vXZOGO/pe3BN//CaQcmM\nbU+Y2VVmtiHVcYiIyL5RMiQikiRm9kT4If7RKp67K3zu9VTEVh0zuyiMq6af4/a2f3cvBDoBb+/B\nZQvCa+bu7evWlZmtNbNrqzh/q5klvv6TwIA69hm5ZE5ERAJKhkREkmsF8AMza1ZxwswygAuA5SmL\nqnovECQeFT/vAC9WOvdJ5YvMLLOuL+Dua929aA/al4XXlNb1mmRz913unr+/X3dP/p5FRKR2SoZE\nRJLrc4KRjR8knPs2UAiMT2xoZjEz+42ZrTCzIjObaWajK7U5zMymmFmhmU0DRlR+QTMbYGZvmNk2\nM8s3s+fMrGNdgg0/5K+t+AGKgK+cc/diM/uDmU02syvMbAlQZGYZZnaGmX1sZlvMbKOZjTOzfgmx\nfWWaXMKoyWgze9/MdprZF4mjT5VHVsxsZMUIVRjDTjObYGaDK/09XBn+Xe40s1fN7HozK6zL30Nt\nKk+TM7NeZva6mW02sx1mNtvMvmdmOcCcsNnMMO5/hdfEzewOM1sZ/ntPN7PTqrjv75vZf8LYfxze\nz+mV4jkj/G+idX3cn4hIY6FkSEQk+R4FLkk4vgR4HPBK7a4HfgncCAwGXgX+bmZDAMysOfAGsBgY\nBtwE/DGxAzPrBHwAfAEMB04CmgP/MLP6fs/vD4wGvgsMAcqApsD/hPGdCBSHr51RS1//FV43BJgJ\nPB8mEjW5E/g5cCiwE3im4okwmXoAuDfs823gtrrf2h57CDDgGIJ/uxuAgnBa4NFhm+MIRtbOCY9/\nBVwH/AI4GHiL4O/qoEp9/4HgPg4CXgp/LqnU5hLgVXffVH+3JCKS/mr75SQiIvtuDPDHcIRkGzAS\n+AlwR6V2NwB/dPcx4fFtZnZMeP5HwLlAFnCxu28HvjCzO4GnE/q4Gpjh7jdWnDCzC4BNBAnKxHq8\nrwzg/EofwF9IbGBmF4evPQSYXENfd7v7uPCaW4GFwKBarrnF3T8Ir/k98I6ZtXX3DQSJ5T/d/Z6w\n7XwzO5yvjtBV53/N7A+VzmUCS2q4pgfwqLvPDI8XJzxXMYK0MRxtq3AD8F/uXvF3dqOZHUuQHF2W\n0O4ed3+t4sDMHgbeM7N27r7ezNoTjDaehoiI7BGNDImIJJm7byYY5bkEuBAY7+5fWS9kZi2AzsDH\nlS7/iC8X6h8EfB4mQhU+rdT+UOAYM9te8UOwbgmgzz7fzFctqTwSYWYHmNkLZrbYzAoSXrt7LX19\nnvB4dfhn+324pj9fT/wm1NJfhf8mSN4Sf+6v5Zo/Ab8PpwjeUTGaV50wgWlNzf/eFb6SELr7R8Ai\n4Pzw1PnAKuDdWmIUEZFKlAyJiOwfjxEUTbgkfLwnKk+nq0mMYCpd5Q/z/YD6rly3I/HAzAz4F5BL\nMLIxAjiMIP6sWvoqSXhccb+1/Y7am2vqYoO7L0z8IRjdqpa7P0CQbD5NkMxMNLOb9vL1K/9776ii\nzSPAxeHji4HH3X1P/jsRERGUDImI7C/vEqyfaQu8VvlJdy8gGN34ZqWnjgJmh4/nAIMTK9MBh1dq\nPxUYCCyr/IHe3bfVw33UpDPQC/idu7/n7nOAVgRrafa3uQSJWKLhyXxBd1/u7n9z97MI1jNdET5V\nHP4ZT2ibD2yk5n/vmjwJHGBmPyEYMXxiH0IXEWm0lAyJiOwH4bf2BwO9aigr/T/ADWZ2Tjjd7A6C\nxfcVRRLGAKXAY2Y20MxOBn5dqY/7gTzgBTMbYWa9zewkM3vIzHLr/ca+Kh/YAlxpZn3N7ATgz+zZ\nyFZ9+TNwhpn91Mz6mdmVBOtqkhKLmd1nZt8Kq8oNBU7my6RmDUFCNNLM2odTIiH4d70lrBZ3QLhO\n6VDgnq+9QCXhuqjXwj7eqTztUkRE6kbJkIjIfuLu28IRoOr8hSAhupugGtx3gTPdfUZ4/XbgdIIp\nb1MJPgjfmNiBu1eMLpUTTFmbRZAgFYU/SePuJcDZBCMwXxCso7kxjGW/cvfxwLUERQpmEBSt+CNB\nSfNkyAQeJBi9+xewDLg0jGUX8LMwnjUE+zZB8G/9F4K/py+AU4HvhCNqdfEowfTDr23qKyIidWOa\nYiwiIo2BmT0IDHP3ytPnGiQzuxD4X6DLnmxiKyIiX1JpbRERSUtmdiPBKM0OgpGhS4CfpjSoemBm\nTQnWZ90EPKhESERk72lkSERE0pKZvQYcSVDdbjHBJqwPNPSqa+Haol8A4wmm1VVVbU5EROpAyZCI\niIiIiDRKKqAgIiIiIiKNkpIhERERERFplJQMiYiIiIhIo6RkSEREREREGiUlQyIiIiIi0igpGRIR\nERERkUZJyZCIiIiIiDRKSoZERERERKRRUjIkIiIiIiKNkpIhERERERFplJQMiYiIiIhIo6RkSERE\nREREGiUlQyIiIiIi0igpGRIRERERkUZJyZCIiIiIiDRKSoZERERERKRRUjIkIiIiIiKNkpIhERER\nERFplJQMiYiIiIhIo6RkSEREREREGiUlQyIiIiIi0igpGRIRERERkUYpI9UB7Km2bdt6z549Ux2G\nSIM3ZcqUDe7eLtVxRJ3ec0Tqh95z6kbvOSL1o67vOQ0uGerZsyeTJ09OdRgiDZ6ZLUt1DA2B3nNE\n6ofec+pG7zki9aOu7zmaJiciIiIiIo2SkiEREREREWmUlAyJiIiIiEijpGRIREREREQaJSVDIiIi\n0miY2Ugzm2dmC83spiqezzazF8LnJ5hZz4Tnbg7PzzOzUxLOP2Zm+Wb2RaW+WpvZ22a2IPyzVTLv\nTUT2nJIhERERaRTMLA7cD5wKDADOMbMBlZpdCmx2977AvcBd4bUDgLOBgcBI4IGwP4AnwnOV3QS8\n6+79gHfDYxGJkAZXWrs2RaVlTF22he5tmtKlZZNUhyMiIpK2ysudzxZvpH+nFrRulpXqcOpiOLDQ\n3RcDmNnzwGhgdkKb0cDt4eOXgfvMzMLzz7t7EbDEzBaG/X3q7h8kjiBV6uu48PGTwHjgxn25AXfn\n00Ub96ULSZG8ppkM7JyX6jCkkrRLhrYXlnLOw59x+xkDuOibvVIdjojsJTMbCfwZiAOPuPsfqml3\nJsEHlsPcfXL4gWQOMC9s8pm7XxW2PZTgG9wmwDjgenf3JN6GSFrKLyjkpSkreXHyCpZt3Mmt3z6I\ny47uneqw6qILsCLheCUworo27l5qZluBNuH5zypd26WW1+vg7mvCx2uBDlU1MrMrgCsAunfvXmOH\n5Q7nPjKhlpeVqPrwV8fTrXXTVIchCdIuGWrVNIuMmLFuW1GqQxGRvZQwleVkgg8ck8xsrLvPrtQu\nF7geqPzJYJG7D6mi6weBy8P24wimtbxZz+GLpCV357PFm3j84yW8OzefsnLn8N6t+fnJB3DKwI6p\nDi/y3N3NrMovX9z9IeAhgGHDhtX4BU3M4IUrDk9ChJJMk5dt5n/emkdBYUmqQ5FK0i4ZisWM9rnZ\n5BcoGRJpwOoylQXgdwTz+X9ZW4dm1glo4e6fhcdPAd9ByZBIjYpKy/jnjDU89tESZq8poHWzLC4/\nujc/PKwbvdo2S3V4e2oV0C3huGt4rqo2K80sA8gDNtbx2srWmVknd18Tvgfl70vwAGbGiN5t9rUb\n2c92FJcCUFKmyQhRk3bJEED7FjnkbytMdRgisvdqncpiZkOBbu7+hplVToZ6mdk0oAC41d0/DPtc\nWanPKqe47MmUFZF0tWLTTp6buJwXJ69gw/ZiDujQnLvOHMzoIV3IyYzX3kE0TQL6mVkvgkTmbODc\nSm3GAhcCnwJnAe+FozpjgTFmdg/QGegHTKzl9Sr6+kP45z/q60akYcmIBTXLSsvKUxyJVJaeyVBu\nNks37kh1GCKSJGYWA+4BLqri6TVAd3ffGK4Res3MBu5J/3syZUUknRSWlPHB/PU8P2kF78/Lx4AT\n+nfgwiN7cFTftgR1BBqucA3QtcBbBOsRH3P3WWZ2BzDZ3ccCjwJPhwUSNhEkTITtXiQYoS4FrnH3\nMgAze46gUEJbM1sJ/NbdHyVIgl40s0uBZcAP9uPtSoRkxIP/dzQyFD1pmQx1aJHDxKWbUh2GiOy9\n2qaj5AKDgPHhh7OOwFgzG+Xuk4EiAHefYmaLgAPC67vW0KdIo7SjqJT35+Xz5hdrGT83nx3FZbTL\nzeba4/ty9vDuaVeZ1d3HEawZTDx3W8LjQuD71Vx7J3BnFefPqab9RuDEfYlX0kNWPBwZKtfIUNSk\naTKUzZadJRSWlDXkoXyRxqzGqSzuvhVoW3FsZuOBG8Jqcu2ATe5eZma9CaayLHb3TWZWYGaHExRQ\nuAD46367I5EIKSgs4d056xg3cy0fzF9PUWk5bZtnMWpIF0YO6siRfdqQGddWhCL1JSP8/6lE0+Qi\nJy2Tofa5OQCs31ak8oUiDVAdp7JU5xjgDjMrAcqBq9y9Yqj4x3xZWvtNVDxBGpFdxWW8M2cdr01b\nxQcL1lNS5nRskcM5w7szclBHDuvZmnisYU+DE4mqjJimyUVVeiZDLbIByN9WqGRIpIGqbSpLpfPH\nJTx+BXilmnaTCabXiTQas1Zv5bGPlvLWrLVsLyqlY4scLjqyJ6cO7sSQri2JKQESSbqKkdZSJUOR\nk5bJUIcWwcjQOpXXFhGRRqqwpIw/vbOAhz9cTNPMOKcN7sh3vtGFEb3aaARIZD/L3F1AQdPkoiYt\nk6H2ucHI0LoCldcWEZHGZ+KSTdz0yucs3rCDHw7rxi2nHURe08xUhyXSaGVqzVBkpWUy1KppFplx\nI3+bRoZERKRxcHcmLNnEU58uZdzMtXRr3YRnLh3BUf3a1nqtiCRXRWnt0nJNk4uatEyGYjGjfW6O\nRoZERCTt7Swu5dVpq3jqk2XMW7eNvCaZXHN8H645vi9Ns9Ly17xIg/PlmiGNDEVN2r5LtsvNJl9r\nhkREJE3lFxTy5KdLeXbCcrbsLGFApxbcfebBnHFIZ5pkaVsJkSjJjAXJULEKKERO2iZDHVpks2TD\njlSHISIiUq9mry7g0Y+WMHbGKkrLnW8N6MClR/XmsJ6tCDchFpGI2T1NTiNDkZPGyVAOny3eVHtD\nERGRiCsrd96Zs47HP17CZ4s30SQzzrnDu3PJUb3o0aZZqsMTkVpozVB0pW0y1D43m627SigsKSMn\nU9MFRESk4SkpK+flKSt5cPwilm/aSZeWTbjltP78cFh3VYcTaUB2T5Mr1chQ1KRvMhTuNbR+W5E2\nXhURkQalpKycv09dyV/fW8jKzbs4pFtLbj61PycP6EBGuBBbRBqOWMyIx4zSciVDUZO2ydCXG68W\nKhkSEZEGobCkjJenrOShDxazfNNODu6ax+9GD+K4A9tpPZBIA5cRM0pVQCFy0jYZ+nLjVVWUExGR\naNuys5hnPlvGE58sZcP2Yg7p1pLfnjGAE/q3VxIkkiay4jFKlAxFTtomQxUjQ/nbtNeQiIhEU0lZ\nOU98vJQ/v7uA7UWlHHdgO646tg8jerVWEiSSZjLiRomqyUVO0pMhM4sDk4FV7n56peeygaeAQ4GN\nwA/dfWl9vG6rpplkxk0jQyIiEkmfLNrAb/8xiwX52zn+wHb8amR/DurUItVhiUiSZMRjWjMUQftj\nZOh6YA5Q1Tv8pcBmd+9rZmcDdwE/rI8XNTPa5+aQX6CRIRERiY4N24u445+zGTtjNd1aN+GRC4Zx\n0oAOqQ5LRJIsM2aaJhdBSU2GzKwr8G3gTuDnVTQZDdwePn4ZuM/MzN3r5b+U9i2yyd+mkSEREUk9\nd2fsjNXcPnYWO4rKuP7Eflx9XB9t/yDSSGRmxDRNLoKSPTL0J+BXQG41z3cBVgC4e6mZbQXaABsS\nG5nZFcAVAN27d6/zi3fIzWHR+u17HrWIiEg9WldQyK9f/YJ35qxjSLeW/M9ZB9OvQ3W/GkUkHama\nXDQlLRkys9OBfHefYmbH7Utf7v4Q8BDAsGHD6vxfUfsW2XyyaEPtDUVERJLA3Xl5ykp+9/psikrL\n+fVpB3HJUb2Ix1QcQaSxyYxrZCiKkjky9E1glJmdBuQALczsGXf/UUKbVUA3YKWZZQB5BIUU6kWH\nFjkUFJZSWFKmaQgiIrJfrd6yi1tencn4eesZ3rM1fzhzML3bNU91WCKSIpnxGKXlGhmKmqQlQ+5+\nM3AzQDgydEOlRAhgLHAh8ClwFvBefa0Xgi/3GsovKKJ7G228KiIiyefuvDh5Bb97fQ5l5c7tZwzg\ngiN6EtNokEijptLa0RTb3y9oZneY2ajw8FGgjZktJCiwcFN9vlb7cK+hddprSKTBMbORZjbPzBaa\nWbXvDWZ2ppm5mQ0Lj082sylmNjP884SEtuPDPqeHP+33x71I41FQWMK1Y6Zx4yszGdwlj7d+egwX\nfbOXEiERITOmaXJRtF82XXX38cD48PFtCecLge8n63U7tPhyZEhEGo5wf7L7gZOBlcAkMxvr7rMr\ntcslKN8/IeH0BuAMd19tZoOAtwiKtVQ4z90nJ/UGpFGavmILP3luKqu3FPKrkQdy1TF9lASJyG4Z\ncaO4VMlQ1Oz3kaH9qUNuODKkvYZEGprhwEJ3X+zuxcDzBKX4K/sdwf5ku/8nd/dp7r46PJwFNAk3\neBZJCnfnkQ8Xc9aDn1BeDi9eeTg/Pq6vEiER+YrMeIwSrRmKnLROhlo2zSQrHtM0OZGGZ3fZ/dBK\nvjq6g5kNBbq5+xs19HMmMNXdE4eHHw+nyP3GzKr8tGpmV5jZZDObvH79+r28BWkMtheVcs2Yqfz+\njTmc0L89b1x3FIf2aJ3qsEQkgjLjRolGhiJnv0yTSxUzo11uNus1TU4krZhZDLgHuKiGNgMJRo2+\nlXD6PHdfFU6vewU4H3iq8rV7W85fGpeF+du48ukpLNmwg5tP7c8Vx/SmmvxaRISMWIzSciVDUZPW\nI0MQrBvSyJBIg1NRdr9C1/BchVxgEDDezJYChwNjE4oodAVeBS5w90UVF7n7qvDPbcAYgul4Invs\n7dnrGH3fx2zZWcIzl47gymP7KBESkRplZsS06WoEpX0y1D43h3UaGRJpaCYB/cysl5llAWcTlOIH\nwN23untbd+/p7j2Bz4BR7j7ZzFoCbwA3ufvHFdeYWYaZtQ0fZwKnA1/sv1uSdDF+Xj4/fnYKfds3\n5/XrjuLIvm1THZKINACZMaNY1eQiJ+2ToQ4tsslXAQWRBsXdS4FrCSrBzQFedPdZlUrzV+daoC9w\nW6US2tnAW2b2OTCdYKTp4eTdhaSjyUs3cdUzU+jXPpenLh1Bp7wmqQ5JRBqIjLhpZCiC0nrNEAR7\nDRUUlrKruIwmWfFUhyMideTu44Bxlc7dVk3b4xIe/x74fTXdHlpf8UnjM3t1ARc/MYnOeU146tLh\n5DXJTHVIItKAZMS1ZiiK0n5kqH1uuNeQ1g2JiMheWrJhBxc8NpHm2Rk8fdkI2jZXtXYR2TNZ8Rgl\nGhmKnLRPhjq0CPYayt+mdUMiIrLnNmwv4sLHJlLuztOXjqBLS02NE5E9lxEzSrRmKHLSPhnqmBck\nQys370xxJCIi0tDsLC7l0icmkb+tkMcuOoy+7ZunOiQRaaAy4qomF0Vpnwz1btuMpllxpi/fkupQ\nRESkASktK+cnY6Yxc9VW7jtnKEO6tUx1SCLSgGXFjZLyctyVEEVJ2idDGfEYQ7q1ZPKyzakORURE\nGgh357djZ/Hu3Hz+3+hBnDSgQ6pDEpEGLiMewx3KypUMRUnaJ0MAh/ZoxZw1BewoKk11KCIi0gA8\nM2E5z05YzlXH9uH8w3ukOhypR2Y20szmmdlCM7upiuezzeyF8PkJZtYz4bmbw/PzzOyU2vo0sxPN\nbGpY4v8jM+ub7PuT6MqIBxszlyoZipRGkwyVO0xfoalyIiJSs22FJdzz73l8s28bfnXKgakOR+qR\nmcWB+4FTgQHAOWY2oFKzS4HN7t4XuBe4K7x2AMEG0AOBkcADZhavpc8HgfPcfQgwBrg1mfcn0ZYZ\nCz52q4hCtDSKZGhoj1aYweSlmionIiI1e/jDJWzeWcJNIw8iFrNUhyP1aziw0N0Xu3sx8DwwulKb\n0cCT4eOXgRPNzMLzz7t7kbsvARaG/dXUpwMtwsd5wOok3Zc0AJkVI0MqohApab/pKkCLnEwO7JDL\n5GWbUh2KiIhE2IbtRTzy4WK+PbgTg7vmpTocqX9dgBUJxyuBEdW1cfdSM9sKtAnPf1bp2i7h4+r6\nvAwYZ2a7gALg8KqCMrMrgCsAunfvvmd3JA1GRlwjQ1HUKEaGIJgqN235Fi1aExGRat3//kKKSsv5\n+bcOSHUokh5+Bpzm7l2Bx4F7qmrk7g+5+zB3H9auXbv9GqDsPxUjQyX6LBopjSYZGtazFduLSpm/\nbluqQxERkQhasWknz362nB8M60qfdtpPKE2tArolHHcNz1XZxswyCKa3bazh2irPm1k74BB3nxCe\nfwE4sn5uQxqizHBkqFQjQ5HSeJKhHq0BVGJbRESq9Kd3FoDBdSf2S3UokjyTgH5m1svMsggKIoyt\n1GYscGH4+CzgPQ82hhkLnB1Wm+sF9AMm1tDnZiDPzCqGGU8G5iTx3iTiNE0umhrFmiGArq2a0C43\nmylLN6lMqoiIfMX8ddv4+7SVXH50bzrlNUl1OJIk4Rqga4G3gDjwmLvPMrM7gMnuPhZ4FHjazBYC\nmwiSG8J2LwKzgVLgGncvA6iqz/D85cArZlZOkBxdsh9vVyImMyzIUqICCpHSaJIhM2NYj1YaGRIR\nka/554zVGHD1sX1SHYokmbuPA8ZVOndbwuNC4PvVXHsncGdd+gzPvwq8uo8hS5rI2D1NTslQlDSa\naXIQFFFYuXkX6woKUx2KiIhEyJw1BfRu15xWzbJSHYqIpKkvCyhomlyUNKpkaFjPYN3QFI0OiYhI\ngjlrtnFQpxa1NxQR2UsVBRRKSpUMRUmjSoYGdm5BTmZMm6+KiMhuW3eWsGrLLg7qlJvqUEQkjWWE\na4ZKVVo7UhpVMpQZj3FI15ZM0earIiISmru2AEAjQyKSVJkZqiYXRY0qGYJgv6FZqwvYVVyW6lBE\nRCQC5qwJkqEBSoZEJIkyYxXJkEaGoqTRJUOH9WxNabnz1/cWEGwbICJRZGYjzWyemS00s5tqaHem\nmbmZDUs4d3N43TwzO2VP+5TGZc6abbRqmkn73OxUhyIiaSwjLKCgTVejpdElQ0f3a8f3D+3KA+MX\ncfPfZ+o/SJEIMrM4cD9wKjAAOMfMBlTRLhe4HpiQcG4Awb4gA4GRwANmFq9rn9L4zF1bwEGdWmBm\nqQ5FRNLYl9Xk9GV8lDS6ZCgeM+4+62CuPb4vz09awZVPT9GUOZHoGQ4sdPfF7l4MPA+MrqLd74C7\ngMR6+aOB5929yN2XAAvD/urapzQiZeXOvHWqJCciyZe5e58hfREfJUlLhswsx8wmmtkMM5tlZv+v\nijbdzex9M5tmZp+b2WnJiqfS63LDKQfyu+8M4r15+Zz7yGfsLC7dHy8tInXTBViRcLwyPLebmQ0F\nurn7G3W8ttY+E/q+wswmm9nk9evX790dSIOwZMMOCkvKlQyJSNJVbLqqAgrRksyRoSLgBHc/BBgC\njDSzwyu1uRV40d2/QTCt5YEkxvM15x/egwfOHcq05Vv40zsL9udLi8g+MLMYcA/wi2T07+4Pufsw\ndx/Wrl27ZLyERERF8QSV1RaRZMsMS2urgEK0JC0Z8sD28DAz/Kn8r+9AxddxecDqZMVTnVMHd+Kc\n4d149KMlzFq9dX+/vIhUbRXQLeG4a3iuQi4wCBhvZkuBw4GxYRGF6q6trU9phOauLSAjZvRt3zzV\noYhImtM0uWhK6pqhcNHydCAfeNvdJ1RqcjvwIzNbCYwDflJNP0mdsnLTyINo1TSTW/4+kzItahOJ\ngklAPzPrZWZZBCPHYyuedPet7t7W3Xu6e0/gM2CUu08O251tZtlm1gvoB0ysrU9pnOas2Uafds3J\nzoinOhTZC2Z2lJldHD5uF/4/LxJJFdXkNDIULUlNhty9zN2HEHwDO9zMBlVqcg7whLt3BU4Dng6n\nv1TuJ6lTVvKaZvKb0wcwY+VWnv50ab33LyJ7xt1LgWuBt4A5BNNpZ5nZHWY2qpZrZwEvArOBfwHX\nhO9FVfaZzPuQ6JuzpkBT5BooM/stcCNwc3gqE3gmdRGJ1KxiZKikXCNDUZKxP17E3beY2fsEZW6/\nSHjq0vAc7v6pmeUAbQlGkvarUYd05pWpq/jjv+dzyqCOdMprsr9DEJEE7j6OYMQ48dxt1bQ9rtLx\nncCddelTGq8tO4tZs7VQxRMaru8C3wCmArj76rDcvkgkZcQq9hnSyFCUJLOaXDszaxk+bgKcDMyt\n1Gw5cGLY5iAgB0hJ6SYz4/ejB1FaXs7tY/VlsYhIupuzZhuAkqGGq9iD3dMdwMyapTgekRrFY4aZ\n1gxFTTKnyXUC3jezzwnm6r/t7q9XmubyC+ByM5sBPAdcFL6xpUT3Nk356UkH8NasdYyZsDxVYYiI\nyH5QUUmuv6bJNVQvmtn/AS3N7HLgHeCRFMckUi0zIzMWo1gjQ5GStGly7v45wfB15fO3JTyeDXwz\nWTHsjcuP7s2nizby27Ff0L9TLkO7t0p1SCIikgRz1hTQtnkW7XNzUh2K7AV3/6OZnQwUAAcCt7n7\n2ykOS6RGGXHTyFDEJLWAQkMUjxl/PnsInfKa8ONnprJ+W1GN7TduL2LJhh37KToREdlT7s7yjTv5\nYP56ikv9QBEkAAAgAElEQVS//BAyZ22Bpsg1YGZ2l7u/7e6/dPcb3P1tM7sr1XGJ1CQjZpSqcnGk\n7JcCCg1Ny6ZZ/O1Hh/K9Bz/mmjFTefayEbsrgCQqK3fOf3QiC9dv575zvsG3BnZMQbQiIgJB0rOt\nqJR1WwtZW1DIik27mLR0ExMWb2T11kIAerdrxm9OH8DRfdsyf912LjqyZ2qDln1xMkE1uUSnVnFO\nJDKyMmIUa2QoUpQMVWNA5xbcdebBXP/8dP5r3Bx+e8bAr7V5ZcpKZq8poHNeDlc/O5W7zzyYMw/t\nmoJoRUQan4LCEqYt38LUZZuZunwzM1ZsoaCw9Ctt2jTL4vDebbi6d2taNs3i3rfnc/Hjkzi0RyuK\nS8tVVrsBMrOrgR8DvcN1yRVygY9TE5VI3WTEYpomFzFKhmowekgXZqzYymMfL2FIt5aMHtJl93Pb\ni0q5+615DO3ekqcuHcGVT0/mFy/NYOuuEi45Snu+iYgkw/ptRfx79lr+9cVaPlm0kbJyJ2ZwYMcW\nnH5IZ3q0bkrHvBw6tMihc14TurVugpntvv6UgR154pMl/OXdhQAM6JSXqluRvTcGeBP4b+CmhPPb\n3H1TakISqZtgzZCmyUWJkqFa3Hxaf75YtZWbXpnJgR1z6d8xmF/+4PiFbNhexCMXDqN5dgaPXXQY\n1z03jTten01peTlXHNMnxZGLiDR8pWXlfL5qKx8t2MBHCzYwedkmyh16tW3GFcf05ui+bTm4W0ua\nZ9ft11lWRowrjunDd7/RlZmrtnBgR40MNTTuvhXYSrBxO2bWnmBrjuZm1tzdVQ5WIisrHqNEa4Yi\nRclQLTLjMe477xuc/pePuOrpKfzj2qMo2FXCwx8u4TtDOjOkW0sAsjPi3H/uUK5+dir3vD2fsw7t\nRutmWSmOXkSkYVq0fjt/fXcB787JZ1tRKWYwqHMe157Qj9MGd+TADrlfGfHZU+1yszmhf4d6jFj2\nNzM7A7gH6EywWXsPYA7w9XntIhGRETdKSjVNLkqUDNVB+9wcHjhvKGc/9Bm/eHE62ZlxYga/Gtn/\nK+0y4jFuHHkgb89ex5OfLOVnJx+QoohFRBqm1Vt28ed3FvDy1JVkZ8QYdUhnju7XjiP6tNEXTFLZ\n74HDgXfc/RtmdjzwoxTHJFKjjFiM0nIlQ1GiZKiOhvVsza3fPojb/zkbgOtO7Efnlk2+1q5v+1xO\nHtCBJz9dypXH9qZplv6KpXEzs58Az7j75lTHItFUWlbOxCWb+Ofna3hl6kpwuOCIHlxzfF/aNs9O\ndXgSXSXuvtHMYmYWc/f3zexPqQ5KpCaZcaNEa4YiRZ/U98CFR/Zk7tptTFyyiauO7V1tu6uO7cPb\ns9fx/MQVKqYgAh2ASWY2FXgMeMvd9ZtAmLJsMy9PWcFbs9axaUcxTTLjfHdIF35yYl+6tmqa6vAk\n+raYWXPgA+BZM8sHtPGfRFpmPEaJqslFipKhPWBm/OHMgykrd+Kx6ueqH9qjFcN7tuaRDxdz/hE9\nqtyjSKSxcPdbzew3wLeAi4H7zOxF4FF3X5Ta6CQVpi7fzL1vz+fDBRtolhXnxIM6cNrgjhx7QHua\nZMVTHZ40HKOBXcDPgPOAPOCOlEYkUgtVk4seJUN7oaZEqMLVx/Xh4icmMXb6au09JI2eu7uZrQXW\nAqVAK+BlM3vb3X+V2ugkWfILCnl/Xj4lZY67U1bu/Gf+et6ft57WzbK45bT+/OjwHppOLHvF3StG\ngcqBJ80sRlBh7tnURSVSs8x4jO2lpbU3lP1Gv4GS5LgD29G/Yy7/98EivvuNLsTqkECJpCMzux64\nANgAPAL80t1Lwg8uCwAlQ2lo7toCLnxsIusKir5yvmXTTG4c2Z8LjuhBszqWwxZJZGYtgGuALsBY\n4O3w+AZgBkqGJMIy4zGNDEWMfhMliZlx5bG9+dkLM/jXrLWcNrhTqkMSSZXWwPfcfVniSXcvN7PT\nUxSTJNGExRu57KnJNM2K88rVR9KtVbDxaTxmNM/OICtDU4dlnzwNbAY+BS4DbgEM+I67T09lYCK1\nyYiZ1gxFjJKhJDr94M786Z0F/PjZqfRr35yTBnTgpIM60Lddc7IyYmRlxGqccufurNlaSKe8nH3a\nT0Mkxd4Edu8KH36re5C7T3D3OakLS5LhX1+s5brnp9GtVROevGS4CiFIMvR298EAZvYIsAbo7u6F\nqQ1LpHYqoBA9SoaSKDMe46WrjuD1GWt4Z846HvpgMQ+OX1SpjdG7bXOG9mjJ0O6tGNQlj/nrtvHB\n/A18uGA9+duKuPXbB3HZ0dVXrxOJuAeBoQnH26s4Jw1YYUkZH8xfz7iZaxg7YzWHdGvJYxceRivt\nCyTJUVLxwN3LzGylEiFpKDLiRmm5pslFiZKhJGufm8MlR/XikqN6sXVnCR8uXM+6giKKS8spLi1n\nZ0kpc9ds443P1/DcxBW7r2vZNJOj+rYlf1sRd781j+MObE/f9s1TeCcie80SS2mH0+Nqfe8xs5HA\nn4E48Ii7/6HS81cRrBMoI0iwrnD32WZ2HvDLhKYHA0PdfbqZjQc6EVSgAviWu+fv/a01bks37ODe\nd+bzzux17Cguo2XTTM4b0YNbTjtIVeEkmQ4xs4LwsQFNwmMjqNfSInWhidQsMx6jpFQjQ1GiZGg/\nymuayekHd67yufJyZ9H67cxctZXe7ZozuEse8ZiRv62Qb937Ab94aQavXHUEGSrTLQ3PYjO7jmA0\nCODHwOKaLjCzOHA/cDKwkmCforHuPjuh2Rh3/1vYfhRwDzDS3Z8lXEBtZoOB1yqtIzjP3SfXw301\nau/Pzee656eBwxmHdOa0wZ04ok8bbSUgSefuyrSlwcqMGyUaGYoU/daKiFjM6Nchl+8N7cqQbi13\nryVqn5vDHaMHMWPFFh7+cEmKoxTZK1cBRwKrCBKbEcAVtVwzHFjo7ovdvRh4nmBPkd3cvSDhsBlQ\n1W+Xc8JrpZ6Ulzv3vbeAS56cRLdWTRl3/dH84cyDOeaAdkqERERqkRGLUao1Q5Gi31wNwBkHd+K0\nwR259+35zF+3LdXhiOwRd89397Pdvb27d3D3c+swNa0LsCLheGV47ivM7BozWwTcDVxXRT8/BJ6r\ndO5xM5tuZr8xVSbZIwvWbePqZ6fwx3/PZ/QhnYNKca1VIEEaFjMbaWbzzGyhmd1UxfPZZvZC+PwE\nM+uZ8NzN4fl5ZnZKbX1a4E4zm29mc8JRcmnEVFo7euo0Tc7M+gAr3b3IzI4jmIP/lLtvSWZwEjAz\nfjd6EBMWf8DPXpjOWYd2ZcvOErbuKqG4rJwjerfh2APb0SInM9WhinyNmeUAlwIDgZyK8+5+yb72\n7e73A/eb2bnArcCFCa87Atjp7l8kXHKeu68ys1zgFeB84KkqYr6CcPSqe/fu+xpmg7Ywfzuvf76a\ncTPXMH/ddjJixm9OH8Al3+ypKpfS4NRxCu6lwGZ372tmZwN3AT80swHA2QTvZZ2Bd8zsgPCa6vq8\nCOgG9A/XS7ZP/l1KlGXGjWKNDEVKXdcMvQIMM7O+wEPAP4AxwGnJCky+qk3zbO787mCuGTOV//fP\n4D07NycDHMZMWE5GzBjRuzWnDOzI6CFdyGuixEgi42lgLnAKcAdwHlBbSe1VBB8gKnQNz1Xneb5c\nk1ThbCqNCrn7qvDPbWY2hmA63teSIXd/iOC9jmHDhjXKr/C27Czmv8bN4cXJKzGDw3q25o7RAxk5\nsCPtW+TU3oFINO2eggtgZhVTcBOTodHA7eHjl4H7wlHk0cDz7l4ELDGzhWF/1NDn1cC57l4OwUh5\nEu9NGgBVk4ueuiZD5e5eambfBf7q7n81s2nJDEy+buSgjkz69UkAtMjJICMeo6zcmbZ8M+/Myeed\nOeu47R+z+O9xc/nu0C5ceERPDuyYW21/5eXOzpIymmsXeEmuvu7+fTMb7e5PhknIh7VcMwnoZ2a9\nCJKgs4FzExuYWT93XxAefhtYkPBcDPgBcHTCuQygpbtvMLNM4HTgnX28t7Tj7rwxcw23j53F5p0l\nXHlMby45qhcdlABJxJjZNr6+VnArMBn4RUVyUklVU3BHVNcm/OyzFWgTnv+s0rUV03er67MPwajS\nd4H1wHUJ71vSCGXEgs9u7q7R9Yio66fgEjM7h2AKyhnhOQ09pEDrSvt2xGPGsJ6tGdazNTed2p8v\nVm3lqU+X8sqUlYyZsJyj+rblltMOYkDnr1Ya/XzlFm54aQbz122nZdNMurduSrfWTTl9cCdOHdxp\nP96RNAIVe4JsMbNBwFqgxqki4QeQa4G3CEprP+bus8zsDmCyu48FrjWzk8L+N5MwRQ44BlhR6cNQ\nNvBWmAjFCRKhh/f99tLHF6u2cs/b83lvbj6Du+TxxMXDGdQlL9VhiVTnTwSJxxiCstpnEyQfU4HH\ngONSFtmXsoFCdx9mZt8jiOvoyo00NbfxyMoIluuXlDlZGUqGoqCuydDFBBWh7nT3JeG3tU8nLyzZ\nW4O65HH3WYdw86kH8fykFTz84WJO/+uHnDeiBz8/+QCaZsf567sLefA/i2jbPIufnXQA+dsKWbF5\nF1OXbebNmWt4/OLhHHtAu/0Sr7vzn/nr+dt/FjF/3XbG//I4rX1KPw+ZWSuCNT1jgebAb2q7yN3H\nAeMqnbst4fH1NVw7Hji80rkdwKF7EnhjMXHJJu5/fyH/mb+e3OwMbv32QVx0ZE+V8peoG+XuhyQc\nP2Rm0939RjO7pZpr6jIFt6LNynBEOQ/YWMu11Z1fCfw9fPwq8HhVQWlqbuOREVYLLikr350YSWrV\nKRkKFwFeBxB+qMl197uSGZjsm1bNsrj6uD6cO7w7974zn6c/W8Y/P19N2+bZLMzfzlmHduU3pw/4\nytqiHUWlnPngJ/xkzFT+ce1R9GrbLGnxlZc7r89cw4PjFzFnTQE5mTEKS8pZtmEng7vqm+h0EU5X\nK3D3zcAHQO8UhyQJVmzayQ0vzWDCkk20aZbFL085kPOP6KEvJKSh2GlmPyBY1wNwFlAYPq4uoah1\nCi7BlzYXAp+Gfb7n7m5mY4ExZnYPQQGFfsBEglGp6vp8DTgeWAIcC8zf+9uVdFDxJZMqykVHXavJ\njQdGhe2nAPlm9rG7/zyJsUk9yGuaye2jBnL28G7c8c/ZLF6/g8cuGsYJ/Tt8rW2z7AwevmAYo+77\niMuenMSr13zzKx+Klm3cwdy121ixaScrN+9iXUEhpw7uxKhDqt5Itjo7ikr52QvT+ffsdfRp14y7\nzzqYvu2b870HPmHVll1KhtJIWD3pV8CLqY5Fvuq9uev42QszKHfn9jMG8MPDutMkS3tZSoNyHvBn\n4AGC5Ocz4Edm1gS4tqoL6jgF91Hg6bBAwiaC5Iaw3YsEhRFKgWvcvQygqj7Dl/wD8KyZ/QzYDlxW\n338J0rBkxcORoXJVlIuKuk6Ty3P3AjO7jKCk9m/N7PNkBib1q3/HFoy5/PBaF+x1a92UB847lPMf\nncBPn5/O3WcdzBufr+Hv01YxY8WXldSbZcVplp3Bm1+s5eMFG7h91MA6fZBasWknlz81mfnrtvGb\n0wdw8ZE9icWMjduLAFizdde+36xEzTtmdgPwArCj4qS7b0pdSI1XaVk5974zn/vfX8SATi148EdD\n6dEmeaPAIskSrgk8o5qnP6rhutqm4BYC36/m2juBO+vSZ3h+C0GBFxHgy5GhEpXXjoy6JkMZZtaJ\noDrTr5MYjyRZXSqXHNGnDb8dNZDfvPYFw34fFNvq3zGXW07rz+G929CtVVNaNs2krNy55+35PDB+\nEdNXbOH+875B3/bVV6+buGQTVz0zhZKycp64eDjHJKxLat0si+yMGKu3KBlKQz8M/7wm4ZyjKXP7\n1c7iUv49ax1PfrqUacu3cPZh3bh91EByMjUaJA2TmbUDLgd6kvB5pj72MBNJloo1Q5omFx11TYbu\nIBj+/djdJ5lZbxLK2Er6Of/wHmwrLGHLzhK+M6TL16rRQVAr/1cj+zOidxt+/sJ0zvjrx/zwsG6c\nPKADw3u1JjMeo6i0jPfnrufVaSt5d04+3Vs35eELh9GnXfOv9GVmdG7ZhNVbC7/2OtKwuXuvVMfQ\nmE1ZtpnnJi7nzZlr2FFcRpeWTfjj9w/hrEO7pjo0kX31D4Iy/e8AZSmORaROMjUyFDl1LaDwEvBS\nwvFi4Myargl3nf+AoKxkBvCyu/+2inY/INjczIEZ7l55IaOkyI+P61undsce0I5x1x/NHf+czfOT\nlvPEJ0tpkZPBsJ6tmbJsM1t3ldC2eTYXHdmTn5zQj7ymVS/O7twyRyNDacjMLqjqvLt/bbNTqT8r\nN+/kv8fN5Y2Za2iencG3D+7EmUO7cljP1sRiKucqaaGpu9+Y6iBE9kRFMqSNV6OjrgUUugJ/Bb4Z\nnvoQuN7dV9ZwWRFwgrtvD/f1+MjM3nT33RuWmVk/4Gbgm+6+2cxq3HtEoqtDixzuP28ou4rL+HDB\nev49ex0Tl2zi2APa8b2hXTiqb9tay/R2ymvCRws27KeIZT86LOFxDnAiwT4gSoaSYGdxKX8bv4j/\n+2AxZvCzkw7g8mN60TRLmytL2nndzE4L1+uINAgZYQGF4lKNDEVFXX87Pk6wqVnFgsIfhedOru4C\nd3eCyikQbNCayddLXV4O3B+W3cXd8+sYj0RUk6w43xrYkW8N7LjH13Zu2YR12wopKSvf/c2JNHzu\n/pPEYzNrCTyfonDSWkFhCT/426fMXbuNUYd05qZT+9O5ZZNUhyWSLNcDt5hZEcHmy0bw8ePr87pF\nIiIzTIY0MhQddf3E2c7dH3f30vDnCaDWXTnNLG5m04F84G13n1CpyQHAAWb2sZl9ZmYjq+nnCjOb\nbGaT169fX8eQpaHpnJeDO6wr0LqhNLcD0DqielZaVs61Y6axMH87j100jL+c8w0lQpLW3D3X3WPu\n3sTdW4THSoQk0nZPk9Oaocio68jQRjP7EfBceHwOwW7MNQrr7w8Jvwl+1cwGufsXlV6/H3AcwY7N\nH5jZ4LAUZWI/2pm5EegUfnBbvaWQrq2apjgaqS9m9k++HBWOAQPQvkP1yt357dhZfDB/PX/43uAq\n9xETSRdm1t/d55rZ0Kqed/ep+zsmkbrKiAXJULGSocioazJ0CcGaoXsJPtR8AlxU1xdx9y1m9j4w\nEkhMhlYCE9y9BFhiZvMJkqNJde1b0keXljmA9hpKQ39MeFwKLKtlvaHsocc+XsqzE5Zz5bG9OXt4\n91SHI5JsPweuAP63iuccOGH/hiNSd7unyam0dmTUtZrcMmBU4jkz+ynwp+quCev/l4SJUBOC9UV3\nVWr2GsEo0+Nm1pZg2tziuocv6aRTXjAytEoV5dLNcmBNuJEhZtbEzHq6+9LUhpUe3py5ht+/MZuR\nAzty4yn9Ux2OSNK5+xXhn8enOhaRPZWxu5qcRoaiYl/KC/2cGpIhoBPwpJnFCabGvOjur5vZHcBk\ndx9LsHfRt8xsNsEeAb9091qn30l6apadQV6TTNZs0ZqhNPMScGTCcVl47rCqm0tdFJeW88d/z+Oh\nDxZzSLeW3PvDISqZLY2OmR3J1zddVaVKiayKkaESjQxFxr4kQzX+1nX3z4FvVHH+toTHTpBU/Xwf\n4pA00ilPew2loQx3L644cPdiM8tKZUAN3ZINO7juuWnMXLWV8w/vwa+/fRA5mfFUhyWyX5nZ00Af\nYDpfbrrqqGy/RJg2XY2efUmGlNJKvevSsgmrt2pkKM2sN7NR4WgwZjYa0IZSe+nNmWv4xUszyIzH\n+L/zD+WUvShjL5ImhgEDwi9WRRqEjJjWDEVNjcmQmW2j6qTHANVslXrXqWUOk5dtTnUYUr+uAp41\ns/vC45XABSmMp8EaO2M1P31+GkO6teS+c4eqdLY0dl8AHYE1qQ5EpK40MhQ9NSZD7p67vwIRgWDj\n1a27SthRVEqz7H0ZuJSocPdFwOFm1jw83l7LJVKFikRoWM/WPHHxYTTN0v8f0ui1BWab2USgqOKk\nu4+q/hKR1PoyGdLIUFTot6lESuewotyarbvo2165eDows/8C7q7YP8zMWgG/cPdbUxtZw6FESKRK\nt6c6AJE9lVFRWlvV5CIjluoARBJ1Tth4VdLGqYkbKbv7ZuC0FMbToLw2bZUSIZFKwkq1t7v7fyr/\npDo2kZpkxjQyFDVKhiRSOuUFG6+qolxaiZtZdsVBuO9Ydg3tK9qNNLN5ZrbQzG6q4vmrzGymmU03\ns4/MbEB4vqeZ7QrPTzezvyVcc2h4zUIz+4uZRboW9TOfLeNnL05neC8lQiKJ3L0MKDezvFTHIrIn\nMjMqCihoZCgq9JtVIqVjXg5mqKJcenkWeNfMHicovnIR8GRNF4Tf+t5PsFnzSmCSmY1199kJzca4\n+9/C9qOAe4CR4XOL3H1IFV0/CFwOTADGhe3f3Mv7SqoHxy/irn/N5cT+7bn/vKEqnS3ydduBmWb2\nNrCj4qS7X5e6kERqlhFTAYWoUTIkkZIZj9E+N1sjQ2nE3e8ysxnASQTVKd8CetRy2XBgobsvBjCz\n54HRwO5kyN0LEto3o5Zy/2bWCWjh7p+Fx08B3yFiyZC7c/db83hw/CJGHdKZ//3BIbsX3IrIV/w9\n/BFpMLTpavQoGZLI6dyyCWu2KhlKM+sIkpXvA0uAV2pp3wVYkXC8EhhRuZGZXUOwaXMWcELCU73M\nbBpQANzq7h+Gfa6s1GeXPbuN5Pvbfxbz4PhFnDuiO78bPYh4LNIz+URSxt1rHGEWiSIzIyNmKqAQ\nIUqGJHI65zVh9pqC2htKpJnZAcA54c8G4AXA3P34+noNd78fuN/MzgVuBS4k2HOku7tvNLNDgdfM\nbOAexn4FcAVA9+7d6yvcWi3M38a9b8/n1EEdufM7g4j4kiaRlDKzfsB/AwOAnIrz7t47ZUGJ1EFG\n3DQyFCGaeyGR07llDqu37EKbijd4cwlGa05396Pc/a9AWR2vXQV0SzjuGp6rzvMEU95w9yJ33xg+\nngIsAg4Ir+9alz7d/SF3H+buw9q1a1fHkPdNWbnzy5c/p1l2nDtGKxESqYPHCdYBlgLHA08Bz6Q0\nIpE6yIzFtGYoQpQMSeR0ymtCUWk5m3YUpzoU2TffIxiled/MHjazEwkKKNTFJKCfmfUysyzgbGBs\nYoPwW+EK3wYWhOfbhQUYMLPeQD9gsbuvAQrM7PCwitwFwD/2/vbq1+MfL2Ha8i3cPmog7XJrLbYn\nItDE3d8lGHFe5u63E7wXiERaRtwo1chQZGianEROxV5Da7YW0qa5PhQ2VO7+GsEUtWYExQ9+CrQ3\nsweBV9393zVcW2pm1xIUW4gDj7n7LDO7A5js7mOBa83sJKAE2EwwRQ7gGOAOMysByoGr3H1T+NyP\ngSeAJgSFEyJRPGHphh388d/zOOmg9ow6pHOqwxFpKIrMLAYsCN8vVgHNUxyTSK0y4zGtGYoQJUMS\nOZ1bBlO/V23ZxaAu2kKioXP3HcAYYIyZtSIoonAjUG0yFF43jqD8deK52xIeX1/Nda9QTYEGd58M\nDNqT+JOtvNz51SufkxmPced3B2t6nEjdXQ80Ba4DfkcwVe7CGq8QiYDMeIziUo0MRYWSIYmc3SND\nKq+ddtx9M/BQ+CPAq9NWMXHJJu4+62A6tMip/QIRAcDdJwGYWbm7X5zqeETqKiOuanJRojVDEjlt\nmmWRlRHTxquS9tyd//tgEf075vL9Q7vWfoGI7GZmR5jZbIJiLZjZIWb2QIrDEqlVZjymNUMRomRI\nIsfM6JyXo41XJe2Nn7+e+eu2c8UxvTU9TmTP/Qk4BaioHjmDYM2gSKRlxIxiVZOLDCVDEkmd8poo\nGZK099B/FtOxRQ5nqGiCyF5x9xWVTtW1fL9IygQjQ0qGokLJkERS55ZNWKNpcpLGZq7cyqeLN3LJ\nUT3JjOutWGQvrDCzIwE3s0wzuwGYk+qgRGoTrBnSNLmo0G9giaQuLXNYV1DIrmJ9ySfp6aEPF5Ob\nncE5w7unOhSRhuoq4BqgC0FZ7SEE5fNFIi0zrk1Xo0TJkETSYb1aU+7w0cINqQ5FpN6t2LSTcTPX\ncM6I7uTmZKY6HJEGyd03uPt57t7B3du7+48INlMWibTMuFGiAgqRoWRIImlErzbkZmfw9uy1qQ5F\npN499vESDLj4mz1THYpIuvl5qgMQqU1GTGuGokTJkERSVkaM4/q35905+ZRpXq2kka07S3hh0gpG\nHdKZTnlNUh2OSLpRWUaJPI0MRYuSIYmskwd0YOOOYqYt35zqUETqzUtTVrCzuIzLju6d6lBE0pE+\nYUrkac1QtGSkOgCR6hx3YDsy48bbs9cxrGfrVIcjss/cnecmLmdo95YM6Nwi1eGINEhmto2qkx4D\n/n97dx5ndV33ffz1njPDvsniiCyiQumICopkgGWudJnSHi5dPsou0zS9Ltv0rtvuNLvbLm3T7iwt\nM4zK8opQc0HSSAQBEVkEBwGFGAHTQZRh/dx/nN/gcRyYhTlzfmfO+/l4nMec3/f8ft/5/DjDd+Zz\nvpu7Wy31yjNlXk0uRdwzZKnVq0sFJx7Wj4eWvlToUMzaxJOrX2Hlxte9gpzZfoiInhHRq5FHz4ho\n8kNeSRMlLZdULenqRl7vLOl3yetzJA3Lee2apHy5pDNbUOePJG1p/V1bR1JRJvcMpYiTIUu106sq\neX7T61Rv8O8QK36/nfsCPTuXc9YxAwsdillJkpQBbgbeD1QB50qqanDaRcArETEcuAn4TnJtFTAZ\nOAqYCNwiKdNUnZLGAAfk9casqGQ3XXXPUFo4GbJUO/XISgAeXubeIStur76xnXufWc8HRw+iWyeP\nUDYrkLFAdUQ8HxHbganApAbnTALuSJ7fDZwqSUn51IjYFhGrgOqkvr3WmSRK3wO+nOf7siJSnnHP\nUJo4GbJUG9SnK0cd3MtD5azo3fPUOrbv3O0hcmaFNQh4Med4bVLW6DkRsROoBfrt49p91Xk5MC0i\n1gVtKSQAABqbSURBVO8rKEkXS5onad7GjRtbdENWfLyAQrrkLRmS1EXSXElPS1oi6Rv7OPcjkiLp\nSjZ7i9OrKlnwwitsfG1boUMxa5X6hROOHdzbCyeYlQhJBwMfA37c1LkRcWtEjImIMQMGDMh/cFZQ\n5WXyAgopks+eoW3AKRFxLDAKmCjpxIYnSeoJXAnMyWMsVsROr6okAh551r1DVpwWvPAKK17a4l4h\ns8JbBwzJOR6clDV6jqRyoDfw8j6u3Vv5aGA4UC1pNdBNUnVb3YgVr4py9wylSd6Sociqn/VekTwa\nS4OvJzs5sS5fsVhxqxrYi0F9unqonBWt3859ke6dMpx97MGFDsWs1D0JjJB0qKROZBdEmNbgnGnA\nhcnzjwKPREQk5ZOT1eYOBUYAc/dWZ0TcGxEHRcSwiBgGvJEsymAlLruaXJD9sbJCy+ucoWSVlYXA\nBuChiJjT4PXjgCERcW8T9XgsbQmTxMSRB/Hoio2s2vR6ocMxa5HarTuYvuifTBo9iO6dvXCCWSEl\nc4AuBx4AlgG/j4glkq6TdE5y2m1Av6QX5yrg6uTaJcDvgaXAX4HLImLX3upsz/uy4lKeyf75vctD\n5VIhr8lQ0kiMIttlPFbSyPrXJJUBNwJfaEY9Hktb4j77nsPolCnjhnuXFToUayfN2LfjEknPSFoo\naVb9UraSTpc0P3ltvqRTcq75W1LnwuRxYL7v47EVG6nbsZuPHNdwjraZFUJE3BcR74iIwyPihqTs\n2oiYljyvi4iPRcTwiBgbEc/nXHtDct07I+L+fdXZyPftke97s+JQkSRDnjeUDu2ymlxEvArMJLsu\nf72ewEjgb8lY2hOBaV5EwRpzYK8ufP7UETy87CUeW+HewY6umXuB3BURRycfuHyX7IcrAJuAsyPi\naLJDXe5scN35ETEqeWzI311kzXpuEz27lHPs4D75/lZmZlYEKjICYLvnDaVCPleTGyCpT/K8K3A6\n8Gz96xFRGxH9c8bSPgGcExHz8hWTFbdPjR/GIf26cd30pZ542PE1uRdIRGzOOexOMicxIp6KiH8m\n5UuArpI6t0PMbxMRzKrexLjD++0ZFmFmZqWtvCybDHnj1XTI52/ngcBMSYvITi58KCKmNxiXa9Zs\nncszfO2sKqo3bOHO2WsKHY7lV3P2AkHSZZJWku0ZuqKRej4CLIiI3HXZf5kMkfvfyUaKebP65TdY\n9+pWJgzvn89vY2ZmRaT+w7Gd/mA3FfI2mzciFpFdVrJh+bV7Of/kfMViHcdpRx7ISSP6c9PDK5g0\n6mD69SjIB/6WEhFxM3CzpPOAr/HmClBIOorsSpVn5FxyfkSsS5b0/yPwSeDXDeuVdDFwMcDQoa1f\nDntW9SYAJozwXEczM8vqlCRDHiaXDh63YUVFEtd+oIo3tu/i+w8uL3Q4lj/N2Qsk11Tgg/UHkgYD\n9wD/HhEr68sjYl3y9TXgLrLD8d6mrRZtmfXcRgb16cqwft1aXYeZmXUs5RkPk0sTJ0NWdEZU9uSi\nCYfy27kv8pen/9n0BVaMmtwLRNKInMOzgOeS8j7AvcDVEfGPnPPLJfVPnlcAHwAW5+sGdu0OHl/5\nMhOG9yfPo/HMzKyI7Bkmt9s9Q2ngZMiK0hfPeCdjDjmAL9+9iGXrNzd9gRWVZu4FcrmkJcleZlfx\n5hC5y8nu+n5tgyW0OwMPJPMYF5Ltafp5vu5h0dpXea1uJ+NHeL6QmZm9qVPSM7TDPUOp4B0ArSh1\nKi/jlguO4+wfz+Kzd85n2uXj6dOtU6HDsjYUEfcB9zUouzbn+ZV7ue6bwDf3Uu3xbRZgE/6RzBca\nf3i/9vqWZmZWBMrLsn0RXhk3HdwzZEXrwJ5d+OkFx7O+ditXTF3onZwtVWZVb+Kog3t5kQ8zM3uL\ncvcMpYqTIStqxw09gOsmjeSxFRv51n3LiHDDYoX3xvadzF/zipfUNjOzt6nw0tqp4mFyVvTOHTuU\n5TWvcdusVWzdsYvrJ40kU+YJ61Y4c1f9ix27ggmeL2RmZg3sSYY8oiUVnAxZh/D1s6vo1inDLX9b\nySuvb+cHk0fRuTxT6LCsRM16bhOdyss4YVjfQodiZmYpUz9MzvsMpYOHyVmHIIkvTzyCr511JPcv\nruFTv3yS1+p2FDosK1GzqjdxwrAD6FLhhNzMzN6qoqx+mJx7htLAyZB1KJ856TBu+sSxzF31Ly74\nxRw2OyGydvav17fzbM1rjPd8ITMza0RFef2mq+4ZSgMnQ9bhfGj0YH56wfEsXb+Zf79trhMia1dr\nXn4dgHdW9ixwJGZmlkb1S2t7mFw6OBmyDun0qkpuPu84Fq+r5cLb53rInLWbmto6AA7q3aXAkZiZ\nWRpVZOp7hjxMLg2cDFmHdcZRB/GT847jmbXZhGjLtp2FDslKQM3mJBnq5WTIzMzernzPanLuGUoD\nJ0PWoU0ceRA/OW80T6+t5fyfP8GmLdsKHZJ1cDWb6+iUKaNv906FDsXMzFKowpuupoqTIevwJo4c\nyM8uOJ7lL73GR3/6+J45HWb5UFNbR2Xvzkje68rMzN6ufjW5HZ4zlApOhqwknFZVyZTPnEjt1h18\n+JbHWbT21UKHZB1UTW0dA3t1LXQYZmaWUuWeM5QqToasZBx/yAHcfek4ulRkmHzrEzywpKbQIVkH\nVLO5jkovnmBmZntRkcwZ2uE5Q6ngZMhKyuEDenDP58Zx+IAefPbO+Xz9z4up27Gr0GFZBxER1NTW\ncVCvzoUOxczMUmpPMrTTPUNp4GTISs6Bvbpw96Xv5tPjD+WO2Wv40C2PU71hS6HDsg7g1Td2sG3n\nbg7q7WFyZmbWuEyZkLyaXFo4GbKS1Lk8w7VnV3HbhWN4aXMdZ/94FnfNeYEIf0pjredltc3MrDkq\nysq8mlxKOBmyknbqkZXcf+VJHHdIH/7XPc/w6V89yYbkD1qzltqTDHnOkJmZ7UNFRuz0anKp4GTI\nSl5lry7c+el38fWzq3h85cuc8YPHuHfR+kKHZUWoptbJkJmZNa08U+altVPCyZAZUFYmPjX+UO69\n4iSG9u3GZXct4PK7FvCyN2m1FqiprUOCA3t6AQUzM9u7iozYsdvD5NLAyZBZjuEH9uCPl47jC6e/\ngweXvMRpNz7Knxeu81yiApA0UdJySdWSrm7k9UskPSNpoaRZkqpyXrsmuW65pDObW+f+qqmto3+P\nzntWCjIzM2tMRabMw+RSwr+xzRqoyJTx+VNHMP2KCQzt150rpy7kP349n7WvvFHo0EqGpAxwM/B+\noAo4NzfZSdwVEUdHxCjgu8CNybVVwGTgKGAicIukTDPr3C81m+u8eIKZmTWpPCMvoJASTobM9uId\nlT3506Xj+NpZRzKreiOn3fgoP3z4Oe9L1D7GAtUR8XxEbAemApNyT4iIzTmH3YH63yqTgKkRsS0i\nVgHVSX1N1rm/amrrPF/IzMyalF1Nzj1DaeBkyGwfMmXiMycdxowvnMypR1Zy08MrOPW/H+W+Z9Z7\n6Fx+DQJezDlem5S9haTLJK0k2zN0RRPXNqvO/eGeITMza47yjNjpnqFUcDJk1gyD+nTl5vOOY+rF\nJ9KzSzmfm7KA838xhxUvvVbo0EpaRNwcEYcDXwG+1lb1SrpY0jxJ8zZu3Nisa7Zu30Xt1h3uGTIz\nsyZVZMq86WpKOBkya4ETD+vH9M9P4PpJR7Hkn5t5/w//zjf+soTarTsKHVpHsw4YknM8OCnbm6nA\nB5u4ttl1RsStETEmIsYMGDCgWQF7w1UzM2uu8kwZ290zlAp5S4YkdZE0V9LTkpZI+kYj51wlaamk\nRZJmSDokX/GYtZXyTBmffPcwZn7xZD5xwhB+9fhqTvn+3/j17NUe/9t2ngRGSDpUUieyCyJMyz1B\n0oicw7OA55Ln04DJkjpLOhQYAcxtTp37o36PoYHuGTIzsyZUlHnT1bTIZ8/QNuCUiDgWGAVMlHRi\ng3OeAsZExDHA3WTH/ZsVhb7dO/GtDx3NXy6fwPADe3Dtn5dw+o2eT9QWImIncDnwALAM+H1ELJF0\nnaRzktMuTz5oWQhcBVyYXLsE+D2wFPgrcFlE7NpbnW0Vc83mrQBUOhkyM7MmZJfW9t8KaVCer4oj\n+9fgluSwInlEg3Nm5hw+AVyQr3jM8mXkoN5MvfhEZi7fwLfvf5bPTVnA6KF9uH7SSEYO6l3o8IpW\nRNwH3Neg7Nqc51fu49obgBuaU2dbqanNbtDrYXJmZtaU8ozY6tVpUyGvc4aSvT0WAhuAhyJizj5O\nvwi4fy/1tHgys1l7ksQpR1Ry/5Xv4bsfOYYX//UG5/xkFtdPX8rr23YWOjxrBzW1W+nZpZzunfP2\nGZOZtYFmbOjcWdLvktfnSBqW81qLNnSWNCUpXyzpdkkV+b4/Kw5eQCE98poMJUNTRpGdqDxW0sjG\nzpN0ATAG+N5e6mnxZGazQsiUiY+fMIQZV53M5LFDuW3WKk7z0LmS4GW1zdKvmZsvXwS8EhHDgZuA\n7yTXtmZD5ynAEcDRQFfgM3m8PSsi5WVeWjst2mU1uYh4FZhJtvF4C0mnAV8FzomIbe0Rj1m+9e5W\nwbc+dDR/vHQcvbtW8LkpC3j/D//OvYvWs3u3G7+OqGbzNi+rbZZ+zdl8eRJwR/L8buBUSaIVGzpH\nxH2RILuQy+A8358ViYpyb7qaFvlcTW6ApD7J867A6cCzDc4ZDfyMbCK0IV+xmBXK8YccwPTPT+DG\njx/L9l27ueyuBZzxg8f4w7wXqfNY4Q6lpnare4bM0q85my/vOSdZeKUW6LePa5usMxke90myi7q8\njacDlJ6KMrHDPUOpkM+eoYHATEmLyC5p+1BETG+wGtT3gB7AHyQtlNRmy9yapUV5powPHzeYh/7r\nvfzkvNGUl4kv3b2Id31rBtdPX8rKjVuarsRSbeeu3Wx8bZuX1TazvbkFeCwi/t7Yi54OUHrKM2Ve\nWjsl8rma3CJgdCPluatBnZav72+WNpky8YFjDuasowcy+/mXmTLnBe54fDW3zVrFuw/rx4XjDuG0\nIyspz3gv5GKzccs2doeX1TYrAs3ZfLn+nLWSyoHewMtNXLvXOiV9HRgAfLYN4rcOoiJTxg4Pm08F\nL3tk1s4kMe7w/ow7vD8bX9vG7+e9yF1zXuCS3yxgYO8unDd2KOe9ayj9enQudKjWTPUbrnqYnFnq\n7dl8mWzCMhk4r8E508juWzYb+CjwSEREMnrlLkk3Agfz5obO2ludkj4DnAmcGhHuBrA9KjLynKGU\ncDJkVkADenbmsvcN55L3Hs6MZS9x5xNr+O+HVnDL31Zy4bhhfPY9h3FA906FDtOasCcZcs+QWapF\nxE5J9ZsvZ4Db6zd0BuZFxDTgNuBOSdXAv8gmNyTn1W/ovJNkQ2eAxupMvuX/A9YAs7NrMPCniLiu\nnW7XUqy8zJuupoWTIbMUyJSJM446iDOOOojqDVv48SPP8bPHVvKbJ9bw6fHDuGjCYfTu5u0p0qpm\ns3uGzIpFMzZ0rgM+tpdrW7Shc0T47yxrlHuG0sOTE8xSZviBPfjh5NE88J/v4aQR/fnRI9W8+9sz\n+D/TlrDm5dcLHZ41oqa2jk6ZMvq6F8/MzJohu+mqe4bSwJ9YmKXUOyp78tMLjmfpPzfzi1nPM2XO\nGu6YvZozqir59PhDGXtoX5JhF1ZgNZvrqOzd2e+HmZk1S3lG7Nod7N4dlJX5d0chORkyS7mqg3tx\n48dH8ZWJR/Dr2auZMucFHljyElUDe/Gp8cM4+9iD6VKRKXSYJa2mto6BvboWOgwzMysSFcnKsTt2\n76ZzmX+HF5KHyZkVicpeXfjSmUcw++pT+b8fPpqdu3fzpbsXMf7bj3DdX5by1AuvkN3k3NpbtmfI\n84XMzKx5KjLZ3iAvolB47hkyKzJdO2U4d+xQJp8whNkrX+aO2av5zRNruP0fqxjStytnH3MwHx8z\nhGH9uxc61JIQEdTU1nFGlZdCNzOz5ikvy/ZH/PIfqzy6oxVOPKwfIwf1bpO6nAyZFSlJjBven3HD\n+1O7dQcPLqnhL4vW87PHnufIgb2cDLWT2q072LZzNwf19jA5MzNrnqF9uwHw/QdXFDiS4vT1s6uc\nDJnZm3p3reBjY4bwsTFDeHnLNrp39n/t9tKnWyeWXndmocMwM7MiclpVJUu+cSa7PLy9VTqXt91M\nH//FZNbB9Ovh4VrtrVsnN6VmZtYy/uAyHbyAgpmZmZmZlSQnQ2ZmZmZmVpKcDJlZKkmaKGm5pGpJ\nVzfy+lWSlkpaJGmGpEOS8vdJWpjzqJP0weS1X0lalfPaqPa+LzMzM0sPD1Y0s9SRlAFuBk4H1gJP\nSpoWEUtzTnsKGBMRb0i6FPgu8ImImAmMSurpC1QDD+Zc96WIuLs97sPMzMzSzT1DZpZGY4HqiHg+\nIrYDU4FJuSdExMyIeCM5fAIY3Eg9HwXuzznPzMzMbA8nQ2aWRoOAF3OO1yZle3MRcH8j5ZOB3zYo\nuyEZWneTpEaX3pN0saR5kuZt3LixJXGbmZlZEXEyZGZFTdIFwBjgew3KBwJHAw/kFF8DHAGcAPQF\nvtJYnRFxa0SMiYgxAwYMyEvcZmZmVnhFN2do/vz5myStaeSl/sCm9o6nmRxb6zi21mtOfIe0RyCt\ntA4YknM8OCl7C0mnAV8F3hsR2xq8/HHgnojYUV8QEeuTp9sk/RL4YlOBuM1pc46tddIcGxR/m5Ma\nbnPanGNrvTTH12ZtTtElQxHR6Me0kuZFxJj2jqc5HFvrOLbWS3t8zfAkMELSoWSToMnAebknSBoN\n/AyYGBEbGqnjXLI9QbnXDIyI9ZIEfBBY3FQgbnPalmNrnTTHBumPr5i4zWlbjq310hxfW8ZWdMmQ\nmXV8EbFT0uVkh7hlgNsjYomk64B5ETGN7LC4HsAfsrkNL0TEOQCShpHtWXq0QdVTJA0ABCwELmmH\n2zEzM7OUcjJkZqkUEfcB9zUouzbn+Wn7uHY1jSy4EBGntGGIZmZmVuQ60gIKtxY6gH1wbK3j2Fov\n7fF1BGn+N3ZsrePYWi/t8XUEaf43dmytk+bYIN3xtVlsioi2qsvMzMzMzKxodKSeITMzMzMzs2Yr\n+mRI0kRJyyVVS7o6BfHcLmmDpMU5ZX0lPSTpueTrAQWKbYikmZKWSloi6cq0xCepi6S5kp5OYvtG\nUn6opDnJ+/s7SZ3aO7acGDOSnpI0PU2xSVot6RlJCyXNS8oK/p52VG5zWhSb25z9i9FtjrnNaVls\nbnP2L8aSbHOKOhmSlAFuBt4PVAHnSqoqbFT8CpjYoOxqYEZEjABmJMeFsBP4QkRUAScClyX/XmmI\nbxtwSkQcC4wCJko6EfgOcFNEDAdeAS4qQGz1rgSW5RynKbb3RcSonGUm0/Cedjhuc1rMbc7+cZtT\n4tzmtJjbnP1Tmm1ORBTtA3g38EDO8TXANSmIaxiwOOd4OTAweT4QWF7oGJNY/gycnrb4gG7AAuBd\nZDfUKm/s/W7nmAYn/9lOAaaTXZo5LbGtBvo3KEvVe9pRHm5z9jtOtznNj8ltjh9uc/Y/Trc5zY+p\nZNucou4ZIrt07os5x2tpZDndFKiM2LPzfQ1QWchgYM8+LKOBOaQkvqR7diGwAXgIWAm8GhE7k1MK\n+f7+APgysDs57kd6YgvgQUnzJV2clKXiPe2A3Oa0ktucFnObY+A2p9Xc5rRYybY53meonUVESCro\nEn6SegB/BP4zIjYru2ElUNj4ImIXMEpSH+Ae4IhCxNGQpA8AGyJivqSTCx1PIyZExDpJBwIPSXo2\n98U0/MxZ4aTh/Xeb0zJuc6yYpeH9d5vTMqXe5hR7z9A6srvM1xuclKXNS5IGAiRfNxQqEEkVZBuI\nKRHxp7TFBxARrwIzyXbJ9pFUn7QX6v0dD5wjaTUwlWwX8g9TEhsRsS75uoFs4zqWlL2nHYjbnBZy\nm9MqbnOsntucFnKb0yol3eYUezL0JDAiWe2iEzAZmFbgmBozDbgweX4h2TGs7U7Zj0ZuA5ZFxI05\nLxU8PkkDkk9KkNSV7BjfZWQbi48WMraIuCYiBkfEMLI/Y49ExPlpiE1Sd0k9658DZwCLScF72kG5\nzWkBtzmt4zbHcrjNaQG3Oa1T8m1OISZCteUD+DdgBdlxl19NQTy/BdYDO8iOr7yI7LjLGcBzwMNA\n3wLFNoHsuMtFwMLk8W9piA84BngqiW0xcG1SfhgwF6gG/gB0LvD7ezIwPS2xJTE8nTyW1P8fSMN7\n2lEfbnNaFJvbnP2P021OiT/c5rQoNrc5+x9nybU5Sio0MzMzMzMrKcU+TM7MzMzMzKxVnAyZmZmZ\nmVlJcjJkZmZmZmYlycmQmZmZmZmVJCdDZmZmZmZWkpwM2dtI2iVpYc7j6jase5ikxW1Vn5kVP7c5\nZtae3OZYrvKmT7EStDUiRhU6CDMrGW5zzKw9uc2xPdwzZM0mabWk70p6RtJcScOT8mGSHpG0SNIM\nSUOT8kpJ90h6OnmMS6rKSPq5pCWSHkx2YkbSFZKWJvVMLdBtmllKuM0xs/bkNqc0ORmyxnRt0H38\niZzXaiPiaOAnwA+Ssh8Dd0TEMcAU4EdJ+Y+ARyPiWOA4sjsHA4wAbo6Io4BXgY8k5VcDo5N6LsnX\nzZlZ6rjNMbP25DbH9lBEFDoGSxlJWyKiRyPlq4FTIuJ5SRVATUT0k7QJGBgRO5Ly9RHRX9JGYHBE\nbMupYxjwUESMSI6/AlRExDcl/RXYAvwP8D8RsSXPt2pmKeA2x8zak9scy+WeIWup2MvzltiW83wX\nb85dOwu4meynK09K8pw2M3ObY2btyW1OiXEyZC31iZyvs5PnjwOTk+fnA39Pns8ALgWQlJHUe2+V\nSioDhkTETOArQG/gbZ/amFnJcZtjZu3JbU6JcUZqjekqaWHO8V8jon7ZyQMkLSL7qce5SdnngV9K\n+hKwEfhUUn4lcKuki8h+MnIpsH4v3zMD/CZpSAT8KCJebbM7MrM0c5tjZu3JbY7t4TlD1mzJWNox\nEbGp0LGYWcfnNsfM2pPbnNLkYXJmZmZmZlaS3DNkZmZmZmYlyT1DZmZmZmZWkpwMmZmZmZlZSXIy\nZGZmZmZmJcnJkJmZmZmZlSQnQ2ZmZmZmVpKcDJmZmZmZWUn6/2gGmN91NxrpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f599d997a20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(14, 4))\n",
    "title = f.suptitle(\"Model Training History\", fontsize=14)\n",
    "f.subplots_adjust(top=0.85, wspace=0.35)\n",
    "\n",
    "ax1.plot(epochs, losses, label='Loss')\n",
    "ax2.plot(epochs, accs,  label='Accuracy')\n",
    "ax3.plot(epochs, lrs, label='Learning Rate')\n",
    "\n",
    "ax1.set_xlabel('Epochs')\n",
    "ax2.set_xlabel('Epochs')\n",
    "ax3.set_xlabel('Epochs')\n",
    "ax1.set_ylabel('Loss')\n",
    "ax2.set_ylabel('Accuracy')\n",
    "ax3.set_ylabel('Learning Rate')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
